Can we differentiate a weed from a crop seedling? The ability to do so effectively can mean better crop yields and better stewardship of the environment. The Aarhus University Signal Processing group, in collaboration with University of Southern Denmark, hasrecently released a dataset containing images of unique plants belonging to 12 species at several growth stages
The goal of the project is to create a classifier capable of determining a plant's species from a photo.
Dataset consist of two file
!pip install opencv-python
Requirement already satisfied: opencv-python in /opt/anaconda3/lib/python3.8/site-packages (4.5.4.60) Requirement already satisfied: numpy>=1.17.3 in /opt/anaconda3/lib/python3.8/site-packages (from opencv-python) (1.19.5)
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import cv2
import os
import seaborn as sns # for data visualization
import tensorflow as tf
import tensorflow.keras as keras
from keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.models import Sequential #sequential api for sequential model
from tensorflow.keras.layers import Dense, Dropout, Flatten #importing different layers
from tensorflow.keras.layers import Conv2D, MaxPooling2D, BatchNormalization, Activation, Input, LeakyReLU,Activation
from tensorflow.keras import backend as K
from tensorflow.keras.utils import to_categorical #to perform one-hot encoding
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPool2D
from tensorflow.keras.optimizers import RMSprop,Adam #optimiers for optimizing the model
from keras.callbacks import EarlyStopping #regularization method to prevent the overfitting
from keras.callbacks import ModelCheckpoint
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras import losses, optimizers
from sklearn.metrics import accuracy_score, confusion_matrix
images = np.load('images.npy')
# Load the labels file of dataset
labels = pd.read_csv('Labels.csv')
seedling_category_count=labels.apply(pd.value_counts)
df_value_count = pd.DataFrame(seedling_category_count)
df_label_count = df_value_count.reset_index()
df_label_count.columns = ['Label', 'Count']
print("Images shape: "+ str(images.shape))
print("There are 4750 color images of size 128*128 of plant seedlings")
print("There are "+ str(labels.shape[0])+" images labels")
Images shape: (4750, 128, 128, 3) There are 4750 color images of size 128*128 of plant seedlings There are 4750 images labels
labels['Label'].value_counts()
Loose Silky-bent 654 Common Chickweed 611 Scentless Mayweed 516 Small-flowered Cranesbill 496 Fat Hen 475 Charlock 390 Sugar beet 385 Cleavers 287 Black-grass 263 Shepherds Purse 231 Maize 221 Common wheat 221 Name: Label, dtype: int64
plt.figure(figsize=(8,8))
plt.pie(df_label_count['Count'],labels=df_label_count['Label'],autopct='%1.2f%%')
plt.show()
plt.figure(figsize=(15,15))
j=0
for label in df_label_count['Label']:
j=j+1
plt.subplot(4,4,j)
index = np.where(labels==label)[0][1]
plt.imshow(images[index])
plt.title(label)
plt.xticks([]), plt.yticks([])
Observation
# Draw given before and after images side by side for comparision
def drawBeforeAfterImages(categories,beforeImages, afterImages,subtitle):
fig=plt.figure(figsize=(20,50))
fig.suptitle(subtitle, fontsize=20)
j=0
for label in categories:
j=j+1
plt.subplot(12,4,j)
index = np.where(labels==label)[0][1]
plt.imshow(beforeImages[index])
plt.title("Before: "+label)
plt.xticks([]), plt.yticks([])
j=j+1
plt.subplot(12,4,j)
index = np.where(labels==label)[0][1]
plt.imshow(afterImages[index])
plt.title("After: "+label)
plt.xticks([]), plt.yticks([])
# Removes the background from image
def removeBackgroundFromImgage(images):
clean_image = []
sets = [];
for idx, image in enumerate(images):
#Gaussian Blur
blurr = cv2.GaussianBlur(image,(5,5),0)
# BGR to HSV image
hsv = cv2.cvtColor(blurr,cv2.COLOR_BGR2HSV)
#GREEN PARAMETERS
lower = (25,40,50)
upper = (75,255,255)
mask = cv2.inRange(hsv,lower,upper)
structure = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(11,11))
#Masking
mask = cv2.morphologyEx(mask,cv2.MORPH_CLOSE,structure)
boolean = mask>0
#Removal of background
new = np.zeros_like(image,np.uint8)
new[boolean] = image[boolean]
clean_image.insert(idx,new)
clean_image = np.asarray(clean_image)
return clean_image
As the images are already in 128*128 size, no resizing of images needed
Now we apply the gaussian blur to each 28x28 pixels array (image) to reduce the noise in the image
gaussian_blur_images = []
for idx, img in enumerate(images):
gaussian_blur_images.insert(idx,cv2.GaussianBlur(img, (5, 5), 0))
categories=df_label_count['Label']
drawBeforeAfterImages(categories,images,gaussian_blur_images,"Gaussian Blur application - Before & After images of seedling")
Data normalization is an important step which ensures that each input parameter (pixel, in this case) has a similar data distribution. This makes convergence faster while training the network
IMG_SIZE=128
normalized_images = []
for idx, img in enumerate(gaussian_blur_images):
normalized_img = img.reshape(-1,IMG_SIZE,IMG_SIZE)
normalized_img = normalized_img/255.0
normalized_img = normalized_img.reshape(IMG_SIZE,IMG_SIZE,-1)
normalized_images.insert(idx,normalized_img)
categories=df_label_count['Label']
drawBeforeAfterImages(categories,gaussian_blur_images,normalized_images,"Normalization application - Before & After images of seedling")
# Visulaising the sample result
training_images = np.asarray(normalized_images)
img_shape=training_images.shape
print("There are "+ str(img_shape[0]) +" images of size "+str(img_shape[1])+"*"+str(img_shape[2])+" having "+str(img_shape[3])+" channel")
There are 4750 images of size 128*128 having 3 channel
clean_image=removeBackgroundFromImgage(images)
categories=df_label_count['Label']
drawBeforeAfterImages(categories,images,clean_image,"Background removal - Before & After images of seedling")
from sklearn.preprocessing import LabelBinarizer
enc = LabelBinarizer()
y = enc.fit_transform(labels)
print("Shape of the data is "+ str(y.shape))
Shape of the data is (4750, 12)
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(training_images,y , test_size=0.2, random_state=2,stratify=y)
generator = ImageDataGenerator(rotation_range = 180,zoom_range = 0.1,width_shift_range = 0.1,height_shift_range = 0.1,horizontal_flip = True,vertical_flip = True)
generator.fit(X_train)
np.random.seed(2)
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(3, 3), input_shape=(128, 128, 3), activation='relu'))
model.add(BatchNormalization())
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
model.add(BatchNormalization())
model.add(Dropout(0.1))
model.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model.add(BatchNormalization())
model.add(Conv2D(filters=128, kernel_size=(3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
model.add(BatchNormalization())
model.add(Dropout(0.1))
model.add(Flatten())
model.add(Dense(12, activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d (Conv2D) (None, 126, 126, 32) 896 _________________________________________________________________ batch_normalization (BatchNo (None, 126, 126, 32) 128 _________________________________________________________________ conv2d_1 (Conv2D) (None, 124, 124, 64) 18496 _________________________________________________________________ max_pooling2d (MaxPooling2D) (None, 62, 62, 64) 0 _________________________________________________________________ batch_normalization_1 (Batch (None, 62, 62, 64) 256 _________________________________________________________________ dropout (Dropout) (None, 62, 62, 64) 0 _________________________________________________________________ conv2d_2 (Conv2D) (None, 60, 60, 64) 36928 _________________________________________________________________ batch_normalization_2 (Batch (None, 60, 60, 64) 256 _________________________________________________________________ conv2d_3 (Conv2D) (None, 58, 58, 128) 73856 _________________________________________________________________ max_pooling2d_1 (MaxPooling2 (None, 29, 29, 128) 0 _________________________________________________________________ batch_normalization_3 (Batch (None, 29, 29, 128) 512 _________________________________________________________________ dropout_1 (Dropout) (None, 29, 29, 128) 0 _________________________________________________________________ flatten (Flatten) (None, 107648) 0 _________________________________________________________________ dense (Dense) (None, 12) 1291788 ================================================================= Total params: 1,423,116 Trainable params: 1,422,540 Non-trainable params: 576 _________________________________________________________________
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5)
mc = ModelCheckpoint('best_model.h5', monitor='val_accuracy', mode='max', verbose=1, save_best_only=True)
history=model.fit(X_train,
y_train, #It expects integers because of the sparse_categorical_crossentropy loss function
epochs=30, #number of iterations over the entire dataset to train on
batch_size=64,validation_split=0.2,callbacks=[es, mc],use_multiprocessing=True)#number of samples per gradient update for training
Epoch 1/30 48/48 [==============================] - 229s 5s/step - loss: 6.2240 - accuracy: 0.4724 - val_loss: 11.4358 - val_accuracy: 0.0921 Epoch 00001: val_accuracy improved from -inf to 0.09211, saving model to best_model.h5 Epoch 2/30 48/48 [==============================] - 210s 4s/step - loss: 3.1467 - accuracy: 0.7493 - val_loss: 26.5171 - val_accuracy: 0.1171 Epoch 00002: val_accuracy improved from 0.09211 to 0.11711, saving model to best_model.h5 Epoch 3/30 48/48 [==============================] - 210s 4s/step - loss: 1.6561 - accuracy: 0.8513 - val_loss: 52.7285 - val_accuracy: 0.1184 Epoch 00003: val_accuracy improved from 0.11711 to 0.11842, saving model to best_model.h5 Epoch 4/30 48/48 [==============================] - 216s 5s/step - loss: 0.8809 - accuracy: 0.9036 - val_loss: 65.2069 - val_accuracy: 0.1184 Epoch 00004: val_accuracy did not improve from 0.11842 Epoch 5/30 48/48 [==============================] - 228s 5s/step - loss: 0.6751 - accuracy: 0.9339 - val_loss: 76.9302 - val_accuracy: 0.1184 Epoch 00005: val_accuracy did not improve from 0.11842 Epoch 6/30 48/48 [==============================] - 230s 5s/step - loss: 0.6264 - accuracy: 0.9408 - val_loss: 105.1188 - val_accuracy: 0.1171 Epoch 00006: val_accuracy did not improve from 0.11842 Epoch 00006: early stopping
Observation
Lets try building another model with clean images ( removal of background from the seedling images)
from sklearn.model_selection import train_test_split
X_train1, X_test1, y_train1, y_test1 = train_test_split(clean_image,y ,test_size=0.2, random_state=2,stratify=y)
np.random.seed(2)
model1 = Sequential()
model1.add(Conv2D(filters=32, kernel_size=(3, 3), input_shape=(128, 128, 3), activation='relu'))
model1.add(BatchNormalization())
model1.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model1.add(MaxPooling2D((2, 2)))
model1.add(BatchNormalization())
model1.add(Dropout(0.1))
model1.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model1.add(MaxPooling2D((2, 2)))
model1.add(BatchNormalization())
model1.add(Dropout(0.1))
model1.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model1.add(MaxPooling2D((2, 2)))
model1.add(BatchNormalization())
model1.add(Dropout(0.1))
model1.add(Conv2D(filters=64, kernel_size=(3, 3), activation='relu'))
model1.add(BatchNormalization())
model1.add(Conv2D(filters=128, kernel_size=(3, 3), activation='relu'))
model1.add(MaxPooling2D((2, 2)))
model1.add(BatchNormalization())
model1.add(Dropout(0.1))
model1.add(Flatten())
model1.add(Dense(12, activation='softmax'))
model1.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model1.summary()
Model: "sequential_1" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= conv2d_4 (Conv2D) (None, 126, 126, 32) 896 _________________________________________________________________ batch_normalization_4 (Batch (None, 126, 126, 32) 128 _________________________________________________________________ conv2d_5 (Conv2D) (None, 124, 124, 64) 18496 _________________________________________________________________ max_pooling2d_2 (MaxPooling2 (None, 62, 62, 64) 0 _________________________________________________________________ batch_normalization_5 (Batch (None, 62, 62, 64) 256 _________________________________________________________________ dropout_2 (Dropout) (None, 62, 62, 64) 0 _________________________________________________________________ conv2d_6 (Conv2D) (None, 60, 60, 64) 36928 _________________________________________________________________ max_pooling2d_3 (MaxPooling2 (None, 30, 30, 64) 0 _________________________________________________________________ batch_normalization_6 (Batch (None, 30, 30, 64) 256 _________________________________________________________________ dropout_3 (Dropout) (None, 30, 30, 64) 0 _________________________________________________________________ conv2d_7 (Conv2D) (None, 28, 28, 64) 36928 _________________________________________________________________ max_pooling2d_4 (MaxPooling2 (None, 14, 14, 64) 0 _________________________________________________________________ batch_normalization_7 (Batch (None, 14, 14, 64) 256 _________________________________________________________________ dropout_4 (Dropout) (None, 14, 14, 64) 0 _________________________________________________________________ conv2d_8 (Conv2D) (None, 12, 12, 64) 36928 _________________________________________________________________ batch_normalization_8 (Batch (None, 12, 12, 64) 256 _________________________________________________________________ conv2d_9 (Conv2D) (None, 10, 10, 128) 73856 _________________________________________________________________ max_pooling2d_5 (MaxPooling2 (None, 5, 5, 128) 0 _________________________________________________________________ batch_normalization_9 (Batch (None, 5, 5, 128) 512 _________________________________________________________________ dropout_5 (Dropout) (None, 5, 5, 128) 0 _________________________________________________________________ flatten_1 (Flatten) (None, 3200) 0 _________________________________________________________________ dense_1 (Dense) (None, 12) 38412 ================================================================= Total params: 244,108 Trainable params: 243,276 Non-trainable params: 832 _________________________________________________________________
generator = ImageDataGenerator(rotation_range = 180,zoom_range = 0.1,width_shift_range = 0.1,height_shift_range = 0.1,horizontal_flip = True,vertical_flip = True)
generator.fit(X_train1)
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=5)
mc = ModelCheckpoint('best_model.h5', monitor='val_accuracy', mode='max', verbose=1, save_best_only=True)
history1=model1.fit(X_train1,
y_train1, #It expects integers because of the sparse_categorical_crossentropy loss function
epochs=30, #number of iterations over the entire dataset to train on
batch_size=64,validation_split=0.2,callbacks=[es, mc],use_multiprocessing=True)#number of samples per gradient update for training
Epoch 1/30 48/48 [==============================] - 180s 4s/step - loss: 1.6430 - accuracy: 0.4941 - val_loss: 7.7425 - val_accuracy: 0.2026 Epoch 00001: val_accuracy improved from -inf to 0.20263, saving model to best_model.h5 Epoch 2/30 48/48 [==============================] - 175s 4s/step - loss: 0.8848 - accuracy: 0.7063 - val_loss: 3.7156 - val_accuracy: 0.3382 Epoch 00002: val_accuracy improved from 0.20263 to 0.33816, saving model to best_model.h5 Epoch 3/30 48/48 [==============================] - 184s 4s/step - loss: 0.6113 - accuracy: 0.7961 - val_loss: 2.2085 - val_accuracy: 0.4697 Epoch 00003: val_accuracy improved from 0.33816 to 0.46974, saving model to best_model.h5 Epoch 4/30 48/48 [==============================] - 174s 4s/step - loss: 0.4538 - accuracy: 0.8451 - val_loss: 1.1748 - val_accuracy: 0.6553 Epoch 00004: val_accuracy improved from 0.46974 to 0.65526, saving model to best_model.h5 Epoch 5/30 48/48 [==============================] - 171s 4s/step - loss: 0.3294 - accuracy: 0.8928 - val_loss: 0.8888 - val_accuracy: 0.7408 Epoch 00005: val_accuracy improved from 0.65526 to 0.74079, saving model to best_model.h5 Epoch 6/30 48/48 [==============================] - 171s 4s/step - loss: 0.2376 - accuracy: 0.9204 - val_loss: 0.9180 - val_accuracy: 0.7408 Epoch 00006: val_accuracy did not improve from 0.74079 Epoch 7/30 48/48 [==============================] - 174s 4s/step - loss: 0.1838 - accuracy: 0.9391 - val_loss: 1.2449 - val_accuracy: 0.6947 Epoch 00007: val_accuracy did not improve from 0.74079 Epoch 8/30 48/48 [==============================] - 171s 4s/step - loss: 0.1645 - accuracy: 0.9474 - val_loss: 0.8764 - val_accuracy: 0.7447 Epoch 00008: val_accuracy improved from 0.74079 to 0.74474, saving model to best_model.h5 Epoch 9/30 48/48 [==============================] - 172s 4s/step - loss: 0.1142 - accuracy: 0.9668 - val_loss: 0.8795 - val_accuracy: 0.7421 Epoch 00009: val_accuracy did not improve from 0.74474 Epoch 10/30 48/48 [==============================] - 186s 4s/step - loss: 0.1102 - accuracy: 0.9641 - val_loss: 0.9239 - val_accuracy: 0.7776 Epoch 00010: val_accuracy improved from 0.74474 to 0.77763, saving model to best_model.h5 Epoch 11/30 48/48 [==============================] - 185s 4s/step - loss: 0.0792 - accuracy: 0.9763 - val_loss: 0.8684 - val_accuracy: 0.7711 Epoch 00011: val_accuracy did not improve from 0.77763 Epoch 12/30 48/48 [==============================] - 194s 4s/step - loss: 0.0724 - accuracy: 0.9773 - val_loss: 0.9941 - val_accuracy: 0.7329 Epoch 00012: val_accuracy did not improve from 0.77763 Epoch 13/30 48/48 [==============================] - 192s 4s/step - loss: 0.0545 - accuracy: 0.9878 - val_loss: 1.0098 - val_accuracy: 0.7395 Epoch 00013: val_accuracy did not improve from 0.77763 Epoch 14/30 48/48 [==============================] - 175s 4s/step - loss: 0.0526 - accuracy: 0.9855 - val_loss: 1.2787 - val_accuracy: 0.6961 Epoch 00014: val_accuracy did not improve from 0.77763 Epoch 15/30 48/48 [==============================] - 175s 4s/step - loss: 0.0470 - accuracy: 0.9895 - val_loss: 0.9905 - val_accuracy: 0.7513 Epoch 00015: val_accuracy did not improve from 0.77763 Epoch 16/30 48/48 [==============================] - 174s 4s/step - loss: 0.0469 - accuracy: 0.9865 - val_loss: 0.8775 - val_accuracy: 0.7763 Epoch 00016: val_accuracy did not improve from 0.77763 Epoch 00016: early stopping
Observation
print(history.history.keys())
# summarize history for accuracy
plt.plot(history.history['accuracy'])
plt.plot(history.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
dict_keys(['loss', 'accuracy', 'val_loss', 'val_accuracy'])
print(history1.history.keys())
# summarize history for accuracy
plt.plot(history1.history['accuracy'])
plt.plot(history1.history['val_accuracy'])
plt.title('model accuracy')
plt.ylabel('accuracy')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
dict_keys(['loss', 'accuracy', 'val_loss', 'val_accuracy'])
model.evaluate(X_test,y_test)
30/30 [==============================] - 20s 670ms/step - loss: 104.3121 - accuracy: 0.1379
[104.31208801269531, 0.13789473474025726]
model1.evaluate(X_test1,y_test1)
30/30 [==============================] - 15s 489ms/step - loss: 0.7204 - accuracy: 0.7937
[0.7203552722930908, 0.793684184551239]
Observation
# Test Prediction
y_test_pred_ln1 = model.predict(X_test)
y_test_pred_classes_ln1 = np.argmax(y_test_pred_ln1, axis=1)
y_test_pred_prob_ln1 = np.max(y_test_pred_ln1, axis=1)
categories=df_label_count['Label']
y_pred = model.predict(X_test)
y_class = np.argmax(y_pred, axis = 1)
y_check = np.argmax(y_test, axis = 1)
cmatrix = confusion_matrix(y_check, y_class)
cmatrix = cmatrix/np.sum(cmatrix, axis=1)
plt.figure(figsize=(10,10))
sns.heatmap(cmatrix, xticklabels=categories, yticklabels=categories, annot=True)
<AxesSubplot:>
plt.figure(figsize=(20,10))
fig, ax = plt.subplots(2,1)
ax[0].plot(history.history['loss'], color='b', label="Training loss")
ax[0].plot(history.history['val_loss'], color='r', label="validation loss",axes =ax[0])
legend = ax[0].legend(loc='best', shadow=True)
ax[1].plot(history.history['accuracy'], color='b', label="Training accuracy")
ax[1].plot(history.history['val_accuracy'], color='r',label="Validation accuracy")
legend = ax[1].legend(loc='best', shadow=True)
<Figure size 1440x720 with 0 Axes>
# Test Prediction
y_test_pred_ln2 = model1.predict(X_test1)
y_test_pred_classes_ln2 = np.argmax(y_test_pred_ln2, axis=1)
y_test_pred_prob_ln2 = np.max(y_test_pred_ln2, axis=1)
categories=df_label_count['Label']
y_pred1 = model1.predict(X_test1)
y_class1 = np.argmax(y_pred1, axis = 1)
y_check1 = np.argmax(y_test1, axis = 1)
cmatrix1 = confusion_matrix(y_check1, y_class1)
cmatrix1 = cmatrix1/np.sum(cmatrix1, axis=1)
plt.figure(figsize=(10,10))
sns.heatmap(cmatrix1, xticklabels=categories, yticklabels=categories, annot=True)
<AxesSubplot:>
Observation
plt.figure(figsize=(20,10))
fig, ax = plt.subplots(2,1)
ax[0].plot(history1.history['loss'], color='b', label="Training loss")
ax[0].plot(history1.history['val_loss'], color='r', label="validation loss",axes =ax[0])
legend = ax[0].legend(loc='best', shadow=True)
ax[1].plot(history1.history['accuracy'], color='b', label="Training accuracy")
ax[1].plot(history1.history['val_accuracy'], color='r',label="Validation accuracy")
legend = ax[1].legend(loc='best', shadow=True)
<Figure size 1440x720 with 0 Axes>